package com.demo.starter.service.impl;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.demo.starter.domain.UploadRet;
import com.demo.starter.entity.*;
import com.demo.starter.repository.*;
import com.demo.starter.service.IArticleCrawlService;
import com.demo.starter.service.IOSSService;
import lombok.extern.log4j.Log4j2;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.web.client.RestTemplate;

import java.io.IOException;
import java.text.ParseException;
import java.util.*;

@Log4j2
@Service
public class ArticleCrawlServiceImpl implements IArticleCrawlService {

    /**
     * 东方头条接口成功状态
     */
    private static final Integer SUCCESS_STATUS = 1;
    /**
     * 东方头条新闻默认状态 1-启用
     */
    private static final Integer DEFAULT_STATUS = 1;
    /**
     * 日期格式 e.g. 2018-07-16
     */
    private static final String DATE_FMT_PATTERN = "yyyy-MM-dd";
    /**
     * 日期格式 e.g. 2018-07-16 14:50:11
     */
    private static final String DATETIME_FMT_PATTERN = "yyyy-MM-dd HH:mm:ss";
    /**
     * 日期格式 e.g. 2018-07-16 14:50
     */
    private static final String DATETIME_PART_FMT_PATTERN = "yyyy-MM-dd HH:mm";
    /**
     * 东方头条新闻详情抓取计划状态 0-初始化
     */
    private static final Integer CRAWL_INIT_STATE = 0;
    /**
     * 东方头条新闻详情抓取计划状态 1-启用
     */
    private static final Integer CRAWL_DONE_STATE = 1;

    @Value("${est.channelCode}")
    private String channelCode;

    @Autowired
    private RestTemplate restTemplate;
    @Autowired
    private EstNewsTypeMapper estNewsTypeMapper;
    @Autowired
    private EstNewsMapper estNewsMapper;
    @Autowired
    private EstNewsCrawlPlanMapper estNewsCrawlPlanMapper;
    @Autowired
    private ArticleCategoryMapper articleCategoryMapper;
    @Autowired
    private ArticleMapper articleMapper;
    @Autowired
    private ArticleBodyMapper articleBodyMapper;

    @Autowired
    private IOSSService qiniuOSSServiceImpl;

    @Override
    public void fetchTypes() {
        log.info("========= 【执行Job]：抓取东方头条类别 ========");
    }

    @Override
    public void fetchDetail() {
        log.info("========= 【执行Job]：抓取东方头条新闻 <start> ========");
        EstNewsTypeExample example = new EstNewsTypeExample();
        example.createCriteria().andIdIsNotNull();
        final List<EstNewsType> estNewsTypes = estNewsTypeMapper.selectByExample(example);
        if (CollectionUtils.isEmpty(estNewsTypes)) {
            log.info("没有类别数据……");
            log.info("========= 【执行Job]：抓取东方头条新闻 <end> ========");
            return;
        }

        for (int i = 0; i < estNewsTypes.size(); i++) {
            requestNewsByType(estNewsTypes.get(i).getType());
        }

    }

    @Override
    public void fetchDetailHtml() {
        log.info("========= 【执行Job]：抓取东方头条新闻详情页 HTML <start> ========");
        List<EstNews> estNewsList = new ArrayList<>();
        Calendar calendar = Calendar.getInstance();
        calendar.add(Calendar.DAY_OF_MONTH, 1);
        final Date tomorrow = calendar.getTime();
        EstNewsCrawlPlanExample ex = new EstNewsCrawlPlanExample();
        ex.createCriteria().andStateEqualTo(CRAWL_INIT_STATE).andExpireAtGreaterThan(tomorrow);
        final List<EstNewsCrawlPlan> estNewsCrawlPlans = estNewsCrawlPlanMapper.selectByExample(ex);
        if (CollectionUtils.isEmpty(estNewsCrawlPlans)) {
            log.info("没有抓取计划……");
            log.info("========= 【执行Job]：抓取东方头条新闻详情页 HTML <end> ========");
            return;
        }
        for (int i = 0; i < estNewsCrawlPlans.size(); i++) {
            final EstNewsCrawlPlan crawlPlan = estNewsCrawlPlans.get(i);
            final EstNews estNews = estNewsMapper.selectByPrimaryKey(crawlPlan.getEstNewsId());
            if (estNews != null) {
                estNewsList.add(estNews);
            }
        }

        if (CollectionUtils.isEmpty(estNewsList)) {
            log.info("抓取计划列表没有可用结果...<返回>");
            log.info("========= 【执行Job]：抓取东方头条新闻详情页 HTML <end> ========");
            return;
        }

        int succeedCount = 0;
        List<Long> errorIds = new ArrayList<>();
        final int size = estNewsList.size();
        for (int i = 0; i < size; i++) {
            final EstNews estNews = estNewsList.get(i);
            final String sourceurl = estNews.getSourceurl();
            try {
                final Document document = Jsoup.connect(sourceurl).get();
                saveArticleFromDocument(document, estNews);
                succeedCount++;
            } catch (IOException e) {
                errorIds.add(estNews.getId());
                log.error("详情页HTML解析出错！！！", e);
                e.printStackTrace();
            }
        }

        log.info("====================== 抓取东方头条新闻详情页 HTML 抓取结果 < ======================");
        log.info("总数量：{}", size);
        List<String> totalIdStrs = new ArrayList<>(size);
        for (EstNews et :
                estNewsList) {
            totalIdStrs.add(et.getId().toString());
        }
        final String ids = StringUtils.join(totalIdStrs.toArray(), ",");

        log.info("estNewsList: {}", ids);
        log.info("成功数量：{}", succeedCount);
        log.info("失败数量：{}", errorIds.size());
        log.info("errorIds: {}", StringUtils.join(errorIds.toArray(), ","));
        log.info("====================== < 抓取东方头条新闻详情页 HTML 抓取结果 ======================");
    }

    /**
     * 保存 est_news 到 article
     *
     * @param doc
     * @param estNews
     */
    private void saveArticleFromDocument(Document doc, EstNews estNews) {
        Elements titleEle = doc.select("h1.title"); // 标题
        Elements contentEle = doc.select("div#content"); // 正文


        final String title = titleEle.html();


        ArticleExample example = new ArticleExample();
        example.createCriteria().andTitleEqualTo(title);
        final List<Article> articles = articleMapper.selectByExample(example);
        if (CollectionUtils.isEmpty(articles)) {
            // 相同标题已存在，则不保存
            return;
        }

        // 图片链接替换为OSS的链接
        List<String> imgSrcList = new ArrayList<>();
        // xpth选择（Chrome console）：$x('//*[@id="main"]//figure//a/img');
        Elements imgEles = doc.select("figure.img").select("img");
        for (Element img :
                imgEles) {
            imgSrcList.add(img.attr("src"));
        }

        Map<String, String> imgSrcMap = new HashMap<>();
        if (!imgSrcList.isEmpty()) {

            for (String url :
                    imgSrcList) {
                try {
                    final UploadRet uploadRet = qiniuOSSServiceImpl.uploadByRemoteURL(url);
                    if (uploadRet.getStatus() != UploadRet.SUCCESS) {
                        log.error("图片转存OSS失败：est_news_id={}, 源图片src={}", estNews.getId(), url);
                        return;
                    }
                    log.debug("转换图片地址为 OSS Url:{}", uploadRet.getUrl());
                    imgSrcMap.put(url, uploadRet.getUrl());
                } catch (IOException e) {
                    log.error("图片转存OSS失败：源图片src=" + url, e);
                    e.printStackTrace();
                }
            }
        }

        String contentHtml = contentEle.html();
        // 替换图片地址
        if (!imgSrcMap.isEmpty()) {
            for (Map.Entry<String, String> entry : imgSrcMap.entrySet()) {
                contentHtml = contentHtml.replace(entry.getKey(), entry.getValue());
            }
        }

        final Date now = new Date(System.currentTimeMillis());
        Article article = new Article();
        article.setTitle(title);
        article.setDescription(title);
        article.setKeyword(title);
        final Date publishDateTime = estNews.getDate();
        final Date publishDate = DateUtils.truncate(publishDateTime, Calendar.DATE);
        article.setPublishDatetime(publishDateTime);
        article.setPublishDate(publishDate);
        article.setSource(estNews.getSource());
        article.setStatus(1);
        String mThumb = estNews.getmThumb();
        try {
            final UploadRet mThumdUploadRet = qiniuOSSServiceImpl.uploadByRemoteURL(mThumb);
            if (mThumdUploadRet.isOk()) {
                mThumb = mThumdUploadRet.getUrl();
            }
        } catch (IOException e) {
            e.printStackTrace();
            log.warn("article.thumb 替换失败，thumb={}", mThumb);
        }
        article.setThumb(mThumb);
        String lbimg = estNews.getLbimg();
        try {
            final UploadRet bigThumdUploadRet = qiniuOSSServiceImpl.uploadByRemoteURL(lbimg);
            if (bigThumdUploadRet.isOk()) {
                lbimg = bigThumdUploadRet.getUrl();
            }
        } catch (IOException e) {
            log.warn("article.bigThumb 替换失败，bigThumb={}", lbimg);
            e.printStackTrace();
        }
        article.setBigThumb(lbimg);
        article.setGmtCreate(now);
        article.setIsGood(false);
        article.setIsTop(false);
        article.setIsHot(false);
        article.setCountView(0);
        article.setCountClick(0);
        article.setPaltform("PC,APP,WAP");
        article.setSource(estNews.getSource());
        article.setSourceUrl(estNews.getSourceurl());
        article.setDefaultView(roundView(2000, 5000));
        article.setStatus(1);

        final String estNewsTypeCode = estNews.getType();
        ArticleCategoryExample categoryExample = new ArticleCategoryExample();
        categoryExample.createCriteria().andSlugEqualTo(estNewsTypeCode);
        final List<ArticleCategory> categories = articleCategoryMapper.selectByExample(categoryExample);
        if (CollectionUtils.isEmpty(categories)) {
            log.warn("无法关联通过type关联article_category，该记录被丢弃est_news.id={}", estNews.getId());
            return;
        }
        article.setCategoryId(categories.get(0).getId());

        // 保存主表
        articleMapper.insert(article);

        ArticleBody articleContent = new ArticleBody();
        articleContent.setArticleId(article.getId());
        articleContent.setContent(contentHtml);
        // 保存正文
        articleBodyMapper.insert(articleContent);

        // 改变抓取计划状态
        EstNewsCrawlPlanExample estNewsCrawlPlanExample = new EstNewsCrawlPlanExample();
        estNewsCrawlPlanExample.createCriteria().andEstNewsIdEqualTo(estNews.getId());
        EstNewsCrawlPlan estNewsCrawlPlan = new EstNewsCrawlPlan();
        estNewsCrawlPlan.setState(CRAWL_DONE_STATE);
        estNewsCrawlPlanMapper.updateByExampleSelective(estNewsCrawlPlan, estNewsCrawlPlanExample);
    }

    private int roundView(int min, int max) {
        Random random = new Random();
        return random.nextInt(max) % (max - min + 1) + min;
    }

    private void requestNewsByType(String type) {
        String url = "https://newswifiapi.dftoutiao.com/jsonnew/refresh?type=" + type + "&qid=" + channelCode;
        try {
            final JSONObject jsonResp = restTemplate.getForEntity(url, JSONObject.class).getBody();
            Integer stat = jsonResp.getInteger("stat");
            String message = jsonResp.getString("message");
            if (!Objects.equals(SUCCESS_STATUS, stat)) {
                log.error("[东方头条按类别获取资讯]失败,{}", message);
                return;
            }

            JSONArray dataArray = jsonResp.getJSONArray("data");
            if (dataArray == null || dataArray.isEmpty()) {
                return;
            }
            for (int i = 0; i < dataArray.size(); i++) {
                JSONObject data = dataArray.getJSONObject(i);
                String m_thumb = null;
                String rowkey = data.getString("rowkey");
                String source = data.getString("source");
                JSONArray imgArr = data.getJSONArray("miniimg");
                if (imgArr != null && !imgArr.isEmpty()) {
                    JSONObject firstImgObj = imgArr.getJSONObject(0);
                    if (firstImgObj != null) {
                        m_thumb = firstImgObj.getString("src");
                    }
                }

                String pk = data.getString("pk");
                boolean isExist = isExist(pk);
                if (isExist) {
                    return;
                }

                EstNews entity = new EstNews();
                entity.setStatus(1);
                entity.setCountView(0L);
                String dateStr = data.getString("date");
                Date date;
                try {
                    date = DateUtils.parseDate(dateStr, DATETIME_FMT_PATTERN);
                } catch (ParseException e) {
                    try {
                        date = DateUtils.parseDate(dateStr, DATETIME_PART_FMT_PATTERN);
                    } catch (ParseException e1) {
                        date = null;
                    }
                }
                if (date == null) {
                    return;
                }
                entity.setDate(date);
                entity.setHotnews(data.getInteger("hotnews"));
                entity.setIspicnews(data.getInteger("ispicnews"));

                String lbimg = null;
                JSONArray lbimgArr = data.getJSONArray(lbimg);
                if (lbimgArr != null && !lbimgArr.isEmpty()) {
                    lbimg = lbimgArr.toJSONString();
                }
                entity.setLbimg(lbimg);

                String miniimg = null;
                JSONArray miniimgArr = data.getJSONArray(lbimg);
                if (miniimgArr != null && !miniimgArr.isEmpty()) {
                    miniimg = miniimgArr.toJSONString();
                }
                entity.setMiniimg(miniimg);


                String miniimg02 = null;
                JSONArray miniimg02Arr = data.getJSONArray(lbimg);
                if (miniimg02Arr != null && !miniimg02Arr.isEmpty()) {
                    miniimg02 = miniimg02Arr.toJSONString();
                }
                entity.setMiniimg02(miniimg02);

                entity.setMiniimg02Size(data.getInteger("miniimg02_size"));
                entity.setMiniimgSize(data.getInteger("miniimg_size"));
                entity.setPicnums(data.getInteger("picnums"));
                entity.setPk(data.getString("pk"));
                entity.setPraisecnt(data.getLong("praisecnt"));
                entity.setRowkey(rowkey);
                entity.setSource(source);
                entity.setSourceurl(data.getString("sourceurl"));
                entity.setTopic(data.getString("topic"));
                entity.setTramplecnt(data.getLong("tramplecnt"));
                entity.setType(type);
                entity.setUrl(url);
                entity.setUrlpv(data.getLong("urlpv"));
                entity.setmThumb(m_thumb);
                final Date now = new Date();
                entity.setGmtCreate(now);
                entity.setGmtModified(now);
                entity.setStatus(DEFAULT_STATUS);

                add(entity);
            }

            log.info("已同步类别{}-{}条新闻...", type, dataArray.size());
        } catch (Exception e) {
            e.printStackTrace();
            log.error("[东方头条按类别获取资讯]失败, type={}", type);
        }
    }

    private void add(EstNews entity) {
        estNewsMapper.insert(entity);
        // 加入抓取计划
        addToCrawlPlan(entity.getId());
    }

    /**
     * 添加抓取计划
     * @param estNewsId
     */
    private void addToCrawlPlan(Long estNewsId) {
        EstNewsCrawlPlanExample ex = new EstNewsCrawlPlanExample();
        ex.createCriteria().andEstNewsIdEqualTo(estNewsId);
        final List<EstNewsCrawlPlan> estNewsCrawlPlans = estNewsCrawlPlanMapper.selectByExample(ex);
        if (!CollectionUtils.isEmpty(estNewsCrawlPlans)) {
            // 如果已有抓取计划，则不重复加入
            return;
        }

        EstNewsCrawlPlan plan = new EstNewsCrawlPlan();
        plan.setEstNewsId(estNewsId);
        Calendar calendar = Calendar.getInstance();
        // 3天有效期
        calendar.add(Calendar.DAY_OF_MONTH, 3);
        final Date later = calendar.getTime();
        plan.setExpireAt(later);
        plan.setState(CRAWL_INIT_STATE);
        estNewsCrawlPlanMapper.insert(plan);
    }

    /**
     * est_news 是否存在
     * @param pk
     * @return
     */
    private boolean isExist(String pk) {
        EstNewsExample ex = new EstNewsExample();
        ex.createCriteria().andPkEqualTo(pk);
        final List<EstNews> estNews = estNewsMapper.selectByExample(ex);
        return CollectionUtils.isEmpty(estNews) ? false : true;
    }

}
